# encoding=utf8
import json
import re
import math
import codecs
from pymongo import MongoClient
from bson import json_util
from urllib.parse import quote_plus
from tqdm import tqdm


user = 'mongouser'
pwd = 'tuanjie123'
host = '118.25.106.14'
port = 27017
url = 'mongodb://%s:%s@%s' % (quote_plus(user), quote_plus(pwd), host)

client = MongoClient(url, port=port)
collection_task_test = client['tagging_test']['task_test']

# db.getCollection('task_test').find({taskKey: 'sentimentAnalysis', canOutput: 1, checkDate: {'$gte': 1654704000000, '$lte': 1654790400000}}).count()
lines = collection_task_test.find(
    {
        'taskKey': 'viewClean',
        'canOutput': 1,
        'checkDate': {'$gte': 1658332800000}  # 8-4: 1659542400000
    },
    {
        'taskData': 1,
        'checkData': 1,
        # 'reCheckData': 1,
        # 'finalData': 1,
        '_id': 0,
    },
    no_cursor_timeout=True
)

# raise Exception('ting')
total = lines.count()
print(total)
# raise Exception("ting")

cut_num = 10000  # 每一万条数据村一个文件
file_index = 1  # 文件标志

save_list = []
success_count = 0
error_count = 0

for idx, line in tqdm(enumerate(lines), total=total):  # lines是动态读取mongo中的数据
    save_list.append(line)

    if (idx + 1) % cut_num == 0:  # 达到存储要求
        # 将数据写入文件
        with codecs.open('./pinglun_json_file/sale_data/json_' + str(file_index) + '.json', 'w', encoding='utf-8') as f:
            json.dump(save_list, f, ensure_ascii=False, indent=4)
        # 清空列表
        save_list = []
        # 文件标志加一
        file_index += 1
        last_save = idx

    # save_list.append(line)
    success_count += 1

# 判断是否有多余数据
if save_list:
    with codecs.open('./pinglun_json_file/sale_data/json_' + str(file_index) + '.json', 'w', encoding='utf-8') as f:
        json.dump(save_list, f, ensure_ascii=False, indent=4)

print(success_count)
print(error_count)

